{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/sklearn/cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ],
      "text/vnd.plotly.v1+html": [
       "<script>requirejs.config({paths: { 'plotly': ['https://cdn.plot.ly/plotly-latest.min']},});if(!window.Plotly) {{require(['plotly'],function(plotly) {window.Plotly=plotly;});}}</script>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import sklearn\n",
    "import xgboost as xgb\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import plotly.offline as py\n",
    "py.init_notebook_mode(connected=True)\n",
    "import plotly.graph_objs as go\n",
    "import plotly.tools as tls\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.model_selection import KFold, RepeatedKFold;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# load in train and test datasets\n",
    "train = pd.read_csv(\"./data/train.csv\")\n",
    "test = pd.read_csv(\"./data/test.csv\")\n",
    "\n",
    "#Store our passenger ID for easy access\n",
    "PassengerId = test['PassengerId']\n",
    "\n",
    "train.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_data = [train, test]\n",
    "\n",
    "train['Name_length'] = train['Name'].apply(len)\n",
    "test['Name_length'] = test['Name'].apply(len)\n",
    "\n",
    "train['Has_Cabin'] = train['Cabin'].apply(lambda x:0 if type(x) == float else 1)\n",
    "test['Has_Cabin'] = test['Cabin'].apply(lambda x:0 if type(x) == float else 1)\n",
    "\n",
    "#Feature engineering\n",
    "#Create new feature FamilySize as a combination of sibsp and Parch\n",
    "for dataset in full_data:\n",
    "    dataset['FamilySize'] = dataset['SibSp'] + dataset['Parch'] + 1\n",
    "# Create new feature IsAlone from FamilySize\n",
    "for dataset in full_data:\n",
    "    dataset['IsAlone'] = 0\n",
    "    dataset.loc[dataset['FamilySize'] == 1, 'IsAlone' ] = 1\n",
    "\n",
    "for dataset in full_data:\n",
    "    dataset['Embarked'] = dataset['Embarked'].fillna('S')\n",
    "for dataset in full_data:\n",
    "    dataset['Fare']= dataset['Fare'].fillna(train['Fare'].median())\n",
    "train['CategoricalFare'] = pd.qcut(train['Fare'],4)\n",
    "\n",
    "for dataset in full_data:\n",
    "    age_avg = dataset['Age'].mean()\n",
    "    age_std = dataset['Age'].std()\n",
    "    age_null_count = dataset['Age'].isnull().sum()\n",
    "    age_null_random_list = np.random.randint(age_avg - age_std, age_avg + age_std, size=age_null_count)\n",
    "    dataset['Age'][np.isnan(dataset['Age'])] = age_null_random_list\n",
    "    dataset['Age'] = dataset['Age'].astype(int)\n",
    "train['CategoricalAge'] = pd.cut(train['Age'], 5)\n",
    "# Define the function to extract titles from passengers' names \n",
    "def get_title(name):\n",
    "    title_search = re.search(' ([A-Za-z]+)\\.', name)\n",
    "    if title_search:\n",
    "        return title_search.group(1)\n",
    "    return \"\"\n",
    "# Create a new feature Title, containing the titles of passenger names\n",
    "for dataset in full_data:\n",
    "    dataset['Title'] = dataset['Name'].apply(get_title)\n",
    "    \n",
    "# Group all non-common titles into one single grouping \"Rare\"\n",
    "for dataset in full_data:\n",
    "    dataset['Title'] = dataset['Title'].replace(['Lady','Countess','Capt','Col','Don',\n",
    "                                                 'Dr','Major','Rev','Sir','Jonkheer','Dona'],'Rare')\n",
    "    dataset['Title'] = dataset['Title'].replace('Mlle', 'Miss')\n",
    "    dataset['Title'] = dataset['Title'].replace('Ms', 'Miss')\n",
    "    dataset['Title'] = dataset['Title'].replace('Mme', 'Mrs')\n",
    "\n",
    "for dataset in full_data:\n",
    "    dataset['Sex'] = dataset['Sex'].map( {'female':0, 'male':1}).astype(int)\n",
    "    title_mapping = {\"Mr\":1 ,\"Miss\":2, \"Mrs\":3 ,\"Master\": 4 ,\"Rare\":5}\n",
    "    dataset['Title'] = dataset['Title'].map(title_mapping)\n",
    "    dataset['Title'] = dataset['Title'].fillna(0)\n",
    "    \n",
    "    dataset['Embarked'] = dataset['Embarked'].map({'S':0, 'C':1 ,'Q':2}).astype(int)\n",
    "    dataset.loc[ dataset['Fare'] <= 7.91, 'Fare']  = 0\n",
    "    dataset.loc[ (dataset['Fare'] > 7.91) & (dataset['Fare'] <= 14.454), 'Fare'] = 1\n",
    "    dataset.loc[ (dataset['Fare'] > 14.454) & (dataset['Fare'] <= 31), 'Fare'] = 2\n",
    "    dataset.loc[ (dataset['Fare'] > 31), 'Fare'] = 3\n",
    "    dataset['Fare'] = dataset['Fare'].astype(int)\n",
    "    \n",
    "    dataset.loc[ dataset['Age'] <= 16, 'Age' ] = 0\n",
    "    dataset.loc[(dataset['Age'] > 16) & (dataset['Age'] <= 32), 'Age'] = 1\n",
    "    dataset.loc[(dataset['Age'] > 32) & (dataset['Age'] <= 48), 'Age'] = 2\n",
    "    dataset.loc[(dataset['Age'] > 48) & (dataset['Age'] <= 64), 'Age'] = 3\n",
    "    dataset.loc[ dataset['Age'] > 64, 'Age'] = 4 ;\n",
    "    \n",
    "# Feature selection\n",
    "drop_elements = ['PassengerId', 'Name', 'Ticket', 'Cabin', 'SibSp']\n",
    "train = train.drop(drop_elements, axis = 1)\n",
    "train = train.drop(['CategoricalAge', 'CategoricalFare'], axis = 1)\n",
    "test  = test.drop(drop_elements, axis = 1)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>Name_length</th>\n",
       "      <th>Has_Cabin</th>\n",
       "      <th>FamilySize</th>\n",
       "      <th>IsAlone</th>\n",
       "      <th>Title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>51</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>44</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Survived  Pclass  Sex  Age  Parch  Fare  Embarked  Name_length  Has_Cabin  \\\n",
       "0         0       3    1    1      0     0         0           23          0   \n",
       "1         1       1    0    2      0     3         1           51          1   \n",
       "2         1       3    0    1      0     1         0           22          0   \n",
       "3         1       1    0    2      0     3         0           44          1   \n",
       "4         0       3    1    2      0     1         0           24          0   \n",
       "\n",
       "   FamilySize  IsAlone  Title  \n",
       "0           2        0      1  \n",
       "1           2        0      3  \n",
       "2           1        1      2  \n",
       "3           2        0      3  \n",
       "4           1        1      1  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SklearnHelper(object):\n",
    "    def __init__(self, clf, seed=0, params=None):\n",
    "        params['random_state'] = seed\n",
    "        self.clf = clf(**params)\n",
    "        \n",
    "    def train(self, x_train, y_train):\n",
    "        self.clf.fit(x_train, y_train)\n",
    "    \n",
    "    def predict(self, x):\n",
    "        return self.clf.predict(x)\n",
    "    \n",
    "    def fit(self, x, y):\n",
    "        return self.clf.fit(x,y)\n",
    "    \n",
    "    def feature_importances(self, x, y):\n",
    "        print(self.clf.fit(x,y).feature_importances_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "ntrain = train.shape[0]\n",
    "ntest = test.shape[0]\n",
    "SEED = 226\n",
    "NFOLDS = 5\n",
    "kf = KFold(n_splits=NFOLDS, shuffle=True ,random_state=SEED) #K-交叉验证\n",
    "\n",
    "#kf = RepeatedKFold(n_splits=NFOLDS, n_repeats =2 ,random_state=SEED) #K-交叉重复验证\n",
    "def get_oof(clf, x_train, y_train, x_test):   #x_train 训练集 y_train 训练集标签 x_test测试集\n",
    "    oof_train = np.zeros((ntrain,))\n",
    "    oof_test = np.zeros((ntest,))\n",
    "    oof_test_skf = np.empty((NFOLDS, ntest))\n",
    "    \n",
    "    for i,(train_index, test_index) in enumerate(kf.split(x_train)): #train_index 被分到训练集中的index  同理test_index\n",
    "        x_tr = x_train[train_index]\n",
    "        y_tr = y_train[train_index]\n",
    "        x_te = x_train[test_index]\n",
    "        y_te = y_train[test_index]\n",
    "        clf.train(x_tr, y_tr)\n",
    "        \n",
    "        oof_train[test_index] = clf.predict(x_te) # 用验证数据集验证训练集训练好的模型\n",
    "        \n",
    "        #oof_test_skf[i, :] = clf.predict(x_test)  # 用测试集测试\n",
    "        print(\"RF K-fold: %d accuracy score is %f \" % (i,accuracy_score(y_te.reshape(-1),clf.predict(x_te).reshape(-1))))\n",
    "    #oof_test[:] = oof_test_skf.mean(axis=0)\n",
    "    oof_test[:] = clf.predict(x_test)\n",
    "    return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Put in our parameters for said classifiers\n",
    "nn_params = {\n",
    "    'hidden_layer_sizes': 10,         # The number of trees in the forest\n",
    "     #'warm_start': True, \n",
    "     'activation': 'relu',\n",
    "    'solver': 'adam',\n",
    "    'alpha': 0.0001,  # regularization term\n",
    "    'batch_size' : 28,    # the max number of features to consider when looking for the best split\n",
    "    'verbose': 1,\n",
    "    'learning_rate_init':0.003,\n",
    "    'max_iter':200,              # Maximum number of iterations. \n",
    "    'early_stopping':False,\n",
    "    'beta_1':0.9,             # use when solver=’adam’\n",
    "    'beta_2':0.999,\n",
    "    'epsilon':1e-8\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create 5 objects that represent our 4 models\n",
    "NN = SklearnHelper(clf=MLPClassifier, seed=SEED, params=nn_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Numpy arrays of train, test and target ( Survived) dataframes to feed into our models\n",
    "y_train = train['Survived'].ravel()\n",
    "train = train.drop(['Survived'], axis=1)\n",
    "x_train = train.values # Creates an array of the train data\n",
    "x_test = test.values # Creats an array of the test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 1, loss = 0.91618456\n",
      "Iteration 2, loss = 0.80141046\n",
      "Iteration 3, loss = 0.76992917\n",
      "Iteration 4, loss = 0.74829168\n",
      "Iteration 5, loss = 0.72742811\n",
      "Iteration 6, loss = 0.70755718\n",
      "Iteration 7, loss = 0.68934297\n",
      "Iteration 8, loss = 0.67127545\n",
      "Iteration 9, loss = 0.65533819\n",
      "Iteration 10, loss = 0.64183796\n",
      "Iteration 11, loss = 0.63123984\n",
      "Iteration 12, loss = 0.61955149\n",
      "Iteration 13, loss = 0.61088637\n",
      "Iteration 14, loss = 0.60358504\n",
      "Iteration 15, loss = 0.59489157\n",
      "Iteration 16, loss = 0.58784471\n",
      "Iteration 17, loss = 0.58101972\n",
      "Iteration 18, loss = 0.57479125\n",
      "Iteration 19, loss = 0.56948513\n",
      "Iteration 20, loss = 0.56321918\n",
      "Iteration 21, loss = 0.55704330\n",
      "Iteration 22, loss = 0.55195045\n",
      "Iteration 23, loss = 0.54773001\n",
      "Iteration 24, loss = 0.54353337\n",
      "Iteration 25, loss = 0.53985127\n",
      "Iteration 26, loss = 0.53378339\n",
      "Iteration 27, loss = 0.53056033\n",
      "Iteration 28, loss = 0.52643931\n",
      "Iteration 29, loss = 0.52310039\n",
      "Iteration 30, loss = 0.52009980\n",
      "Iteration 31, loss = 0.51645969\n",
      "Iteration 32, loss = 0.51411274\n",
      "Iteration 33, loss = 0.51038847\n",
      "Iteration 34, loss = 0.50870522\n",
      "Iteration 35, loss = 0.50554082\n",
      "Iteration 36, loss = 0.50265808\n",
      "Iteration 37, loss = 0.50015565\n",
      "Iteration 38, loss = 0.49759500\n",
      "Iteration 39, loss = 0.49741668\n",
      "Iteration 40, loss = 0.49346231\n",
      "Iteration 41, loss = 0.49209335\n",
      "Iteration 42, loss = 0.48940349\n",
      "Iteration 43, loss = 0.48736271\n",
      "Iteration 44, loss = 0.48598330\n",
      "Iteration 45, loss = 0.48406647\n",
      "Iteration 46, loss = 0.48205963\n",
      "Iteration 47, loss = 0.48025293\n",
      "Iteration 48, loss = 0.47862232\n",
      "Iteration 49, loss = 0.47728965\n",
      "Iteration 50, loss = 0.47584743\n",
      "Iteration 51, loss = 0.47406134\n",
      "Iteration 52, loss = 0.47239433\n",
      "Iteration 53, loss = 0.47158101\n",
      "Iteration 54, loss = 0.46980591\n",
      "Iteration 55, loss = 0.46992542\n",
      "Iteration 56, loss = 0.46727829\n",
      "Iteration 57, loss = 0.46766949\n",
      "Iteration 58, loss = 0.46651912\n",
      "Iteration 59, loss = 0.46365262\n",
      "Iteration 60, loss = 0.46278930\n",
      "Iteration 61, loss = 0.46237381\n",
      "Iteration 62, loss = 0.46059528\n",
      "Iteration 63, loss = 0.46046626\n",
      "Iteration 64, loss = 0.45865017\n",
      "Iteration 65, loss = 0.45749378\n",
      "Iteration 66, loss = 0.45682689\n",
      "Iteration 67, loss = 0.45629200\n",
      "Iteration 68, loss = 0.45557775\n",
      "Iteration 69, loss = 0.45467536\n",
      "Iteration 70, loss = 0.45386038\n",
      "Iteration 71, loss = 0.45234145\n",
      "Iteration 72, loss = 0.45195194\n",
      "Iteration 73, loss = 0.45159092\n",
      "Iteration 74, loss = 0.45063827\n",
      "Iteration 75, loss = 0.44927482\n",
      "Iteration 76, loss = 0.44937226\n",
      "Iteration 77, loss = 0.44906329\n",
      "Iteration 78, loss = 0.44791366\n",
      "Iteration 79, loss = 0.44746887\n",
      "Iteration 80, loss = 0.44600222\n",
      "Iteration 81, loss = 0.44632014\n",
      "Iteration 82, loss = 0.44503118\n",
      "Iteration 83, loss = 0.44521857\n",
      "Iteration 84, loss = 0.44382158\n",
      "Iteration 85, loss = 0.44307524\n",
      "Iteration 86, loss = 0.44310429\n",
      "Iteration 87, loss = 0.44279873\n",
      "Iteration 88, loss = 0.44157819\n",
      "Iteration 89, loss = 0.44188775\n",
      "Iteration 90, loss = 0.44077297\n",
      "Iteration 91, loss = 0.43989958\n",
      "Iteration 92, loss = 0.44037766\n",
      "Iteration 93, loss = 0.43955609\n",
      "Iteration 94, loss = 0.43795825\n",
      "Iteration 95, loss = 0.43843786\n",
      "Iteration 96, loss = 0.43807898\n",
      "Iteration 97, loss = 0.43712349\n",
      "Iteration 98, loss = 0.43658006\n",
      "Iteration 99, loss = 0.43758768\n",
      "Iteration 100, loss = 0.43591335\n",
      "Iteration 101, loss = 0.43536200\n",
      "Iteration 102, loss = 0.43524626\n",
      "Iteration 103, loss = 0.43483091\n",
      "Iteration 104, loss = 0.43409523\n",
      "Iteration 105, loss = 0.43364266\n",
      "Iteration 106, loss = 0.43355379\n",
      "Iteration 107, loss = 0.43293604\n",
      "Iteration 108, loss = 0.43233711\n",
      "Iteration 109, loss = 0.43184334\n",
      "Iteration 110, loss = 0.43177406\n",
      "Iteration 111, loss = 0.43129134\n",
      "Iteration 112, loss = 0.43116560\n",
      "Iteration 113, loss = 0.43088979\n",
      "Iteration 114, loss = 0.43026103\n",
      "Iteration 115, loss = 0.42944789\n",
      "Iteration 116, loss = 0.42983808\n",
      "Iteration 117, loss = 0.42940803\n",
      "Iteration 118, loss = 0.42865644\n",
      "Iteration 119, loss = 0.42898406\n",
      "Iteration 120, loss = 0.42863179\n",
      "Iteration 121, loss = 0.42849388\n",
      "Iteration 122, loss = 0.42777455\n",
      "Iteration 123, loss = 0.42774051\n",
      "Iteration 124, loss = 0.42695926\n",
      "Iteration 125, loss = 0.42651982\n",
      "Iteration 126, loss = 0.42650111\n",
      "Iteration 127, loss = 0.42642639\n",
      "Iteration 128, loss = 0.42809610\n",
      "Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.\n",
      "RF K-fold: 0 accuracy score is 0.743017 \n",
      "Iteration 1, loss = 0.94342692\n",
      "Iteration 2, loss = 0.81133889\n",
      "Iteration 3, loss = 0.77222763\n",
      "Iteration 4, loss = 0.75433630\n",
      "Iteration 5, loss = 0.73487237\n",
      "Iteration 6, loss = 0.71464567\n",
      "Iteration 7, loss = 0.69832994\n",
      "Iteration 8, loss = 0.68124143\n",
      "Iteration 9, loss = 0.66659189\n",
      "Iteration 10, loss = 0.65384046\n",
      "Iteration 11, loss = 0.64375549\n",
      "Iteration 12, loss = 0.63176343\n",
      "Iteration 13, loss = 0.62300600\n",
      "Iteration 14, loss = 0.61434427\n",
      "Iteration 15, loss = 0.60759489\n",
      "Iteration 16, loss = 0.60113895\n",
      "Iteration 17, loss = 0.59386432\n",
      "Iteration 18, loss = 0.58855898\n",
      "Iteration 19, loss = 0.58325599\n",
      "Iteration 20, loss = 0.57682372\n",
      "Iteration 21, loss = 0.57239102\n",
      "Iteration 22, loss = 0.56701968\n",
      "Iteration 23, loss = 0.56236795\n",
      "Iteration 24, loss = 0.55706554\n",
      "Iteration 25, loss = 0.55404621\n",
      "Iteration 26, loss = 0.54959795\n",
      "Iteration 27, loss = 0.54669867\n",
      "Iteration 28, loss = 0.54169253\n",
      "Iteration 29, loss = 0.53891156\n",
      "Iteration 30, loss = 0.53534770\n",
      "Iteration 31, loss = 0.53189288\n",
      "Iteration 32, loss = 0.52993499\n",
      "Iteration 33, loss = 0.52729053\n",
      "Iteration 34, loss = 0.52421531\n",
      "Iteration 35, loss = 0.52084439\n",
      "Iteration 36, loss = 0.51825093\n",
      "Iteration 37, loss = 0.51622528\n",
      "Iteration 38, loss = 0.51301032\n",
      "Iteration 39, loss = 0.51103560\n",
      "Iteration 40, loss = 0.50877320\n",
      "Iteration 41, loss = 0.50765427\n",
      "Iteration 42, loss = 0.50442742\n",
      "Iteration 43, loss = 0.50322659\n",
      "Iteration 44, loss = 0.50099416\n",
      "Iteration 45, loss = 0.49856978\n",
      "Iteration 46, loss = 0.49702009\n",
      "Iteration 47, loss = 0.49613988\n",
      "Iteration 48, loss = 0.49356593\n",
      "Iteration 49, loss = 0.49203316\n",
      "Iteration 50, loss = 0.49067065\n",
      "Iteration 51, loss = 0.48889592\n",
      "Iteration 52, loss = 0.48984416\n",
      "Iteration 53, loss = 0.48632994\n",
      "Iteration 54, loss = 0.48436151\n",
      "Iteration 55, loss = 0.48310709\n",
      "Iteration 56, loss = 0.48167896\n",
      "Iteration 57, loss = 0.48048090\n",
      "Iteration 58, loss = 0.47943795\n",
      "Iteration 59, loss = 0.47806916\n",
      "Iteration 60, loss = 0.47694194\n",
      "Iteration 61, loss = 0.47608660\n",
      "Iteration 62, loss = 0.47504301\n",
      "Iteration 63, loss = 0.47400436\n",
      "Iteration 64, loss = 0.47281509\n",
      "Iteration 65, loss = 0.47178388\n",
      "Iteration 66, loss = 0.47069975\n",
      "Iteration 67, loss = 0.47097692\n",
      "Iteration 68, loss = 0.47079813\n",
      "Iteration 69, loss = 0.46845349\n",
      "Iteration 70, loss = 0.46757644\n",
      "Iteration 71, loss = 0.46671283\n",
      "Iteration 72, loss = 0.46620240\n",
      "Iteration 73, loss = 0.46610608\n",
      "Iteration 74, loss = 0.46377337\n",
      "Iteration 75, loss = 0.46343903\n",
      "Iteration 76, loss = 0.46280694\n",
      "Iteration 77, loss = 0.46175743\n",
      "Iteration 78, loss = 0.46130109\n",
      "Iteration 79, loss = 0.46057733\n",
      "Iteration 80, loss = 0.45935653\n",
      "Iteration 81, loss = 0.45897395\n",
      "Iteration 82, loss = 0.45836939\n",
      "Iteration 83, loss = 0.45763030\n",
      "Iteration 84, loss = 0.45674483\n",
      "Iteration 85, loss = 0.45606829\n",
      "Iteration 86, loss = 0.45591269\n",
      "Iteration 87, loss = 0.45515781\n",
      "Iteration 88, loss = 0.45522479\n",
      "Iteration 89, loss = 0.45402819\n",
      "Iteration 90, loss = 0.45399746\n",
      "Iteration 91, loss = 0.45389443\n",
      "Iteration 92, loss = 0.45487709\n",
      "Iteration 93, loss = 0.45308569\n",
      "Iteration 94, loss = 0.45106360\n",
      "Iteration 95, loss = 0.45150022\n",
      "Iteration 96, loss = 0.45061695\n",
      "Iteration 97, loss = 0.44977932\n",
      "Iteration 98, loss = 0.44948256\n",
      "Iteration 99, loss = 0.44916706\n",
      "Iteration 100, loss = 0.44887183\n",
      "Iteration 101, loss = 0.44822081\n",
      "Iteration 102, loss = 0.44815030\n",
      "Iteration 103, loss = 0.44717489\n",
      "Iteration 104, loss = 0.44679154\n",
      "Iteration 105, loss = 0.44665008\n",
      "Iteration 106, loss = 0.44722802\n",
      "Iteration 107, loss = 0.44603173\n",
      "Iteration 108, loss = 0.44603607\n",
      "Iteration 109, loss = 0.44590826\n",
      "Iteration 110, loss = 0.44633493\n",
      "Iteration 111, loss = 0.44405557\n",
      "Iteration 112, loss = 0.44467048\n",
      "Iteration 113, loss = 0.44377120\n",
      "Iteration 114, loss = 0.44345197\n",
      "Iteration 115, loss = 0.44334224\n",
      "Iteration 116, loss = 0.44262956\n",
      "Iteration 117, loss = 0.44283646\n",
      "Iteration 118, loss = 0.44243435\n",
      "Iteration 119, loss = 0.44200651\n",
      "Iteration 120, loss = 0.44126878\n",
      "Iteration 121, loss = 0.44265719\n",
      "Iteration 122, loss = 0.44079887\n",
      "Iteration 123, loss = 0.44139436\n",
      "Iteration 124, loss = 0.44061963\n",
      "Iteration 125, loss = 0.44017060\n",
      "Iteration 126, loss = 0.44023246\n",
      "Iteration 127, loss = 0.44049709\n",
      "Iteration 128, loss = 0.43926002\n",
      "Iteration 129, loss = 0.43911711\n",
      "Iteration 130, loss = 0.44017364\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 131, loss = 0.43852155\n",
      "Iteration 132, loss = 0.43853443\n",
      "Iteration 133, loss = 0.44098704\n",
      "Iteration 134, loss = 0.43872602\n",
      "Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.\n",
      "RF K-fold: 1 accuracy score is 0.820225 \n",
      "Iteration 1, loss = 0.92100679\n",
      "Iteration 2, loss = 0.80010646\n",
      "Iteration 3, loss = 0.76658937\n",
      "Iteration 4, loss = 0.74625826\n",
      "Iteration 5, loss = 0.72631712\n",
      "Iteration 6, loss = 0.70681588\n",
      "Iteration 7, loss = 0.68986649\n",
      "Iteration 8, loss = 0.67348302\n",
      "Iteration 9, loss = 0.65845883\n",
      "Iteration 10, loss = 0.64537120\n",
      "Iteration 11, loss = 0.63442864\n",
      "Iteration 12, loss = 0.62390336\n",
      "Iteration 13, loss = 0.61588062\n",
      "Iteration 14, loss = 0.61021485\n",
      "Iteration 15, loss = 0.60308820\n",
      "Iteration 16, loss = 0.59380047\n",
      "Iteration 17, loss = 0.58764130\n",
      "Iteration 18, loss = 0.58264537\n",
      "Iteration 19, loss = 0.57606121\n",
      "Iteration 20, loss = 0.57100423\n",
      "Iteration 21, loss = 0.56592812\n",
      "Iteration 22, loss = 0.56085036\n",
      "Iteration 23, loss = 0.55683323\n",
      "Iteration 24, loss = 0.55571751\n",
      "Iteration 25, loss = 0.54872976\n",
      "Iteration 26, loss = 0.54449689\n",
      "Iteration 27, loss = 0.54135103\n",
      "Iteration 28, loss = 0.53842902\n",
      "Iteration 29, loss = 0.53414008\n",
      "Iteration 30, loss = 0.53158311\n",
      "Iteration 31, loss = 0.52817036\n",
      "Iteration 32, loss = 0.52470571\n",
      "Iteration 33, loss = 0.52232824\n",
      "Iteration 34, loss = 0.51942474\n",
      "Iteration 35, loss = 0.51668287\n",
      "Iteration 36, loss = 0.51462223\n",
      "Iteration 37, loss = 0.51236737\n",
      "Iteration 38, loss = 0.51019445\n",
      "Iteration 39, loss = 0.50783208\n",
      "Iteration 40, loss = 0.50556754\n",
      "Iteration 41, loss = 0.50470048\n",
      "Iteration 42, loss = 0.50194785\n",
      "Iteration 43, loss = 0.49949290\n",
      "Iteration 44, loss = 0.49783062\n",
      "Iteration 45, loss = 0.49697626\n",
      "Iteration 46, loss = 0.49747608\n",
      "Iteration 47, loss = 0.49368345\n",
      "Iteration 48, loss = 0.49118066\n",
      "Iteration 49, loss = 0.48986581\n",
      "Iteration 50, loss = 0.48865137\n",
      "Iteration 51, loss = 0.48667713\n",
      "Iteration 52, loss = 0.48654489\n",
      "Iteration 53, loss = 0.48440990\n",
      "Iteration 54, loss = 0.48327607\n",
      "Iteration 55, loss = 0.48132551\n",
      "Iteration 56, loss = 0.48087387\n",
      "Iteration 57, loss = 0.47833793\n",
      "Iteration 58, loss = 0.47753891\n",
      "Iteration 59, loss = 0.47595924\n",
      "Iteration 60, loss = 0.47577858\n",
      "Iteration 61, loss = 0.47377557\n",
      "Iteration 62, loss = 0.47271277\n",
      "Iteration 63, loss = 0.47152800\n",
      "Iteration 64, loss = 0.47051752\n",
      "Iteration 65, loss = 0.46936896\n",
      "Iteration 66, loss = 0.46880452\n",
      "Iteration 67, loss = 0.46835055\n",
      "Iteration 68, loss = 0.46670922\n",
      "Iteration 69, loss = 0.46597140\n",
      "Iteration 70, loss = 0.46543650\n",
      "Iteration 71, loss = 0.46373396\n",
      "Iteration 72, loss = 0.46333815\n",
      "Iteration 73, loss = 0.46200586\n",
      "Iteration 74, loss = 0.46172452\n",
      "Iteration 75, loss = 0.46170787\n",
      "Iteration 76, loss = 0.46018135\n",
      "Iteration 77, loss = 0.45953119\n",
      "Iteration 78, loss = 0.45809823\n",
      "Iteration 79, loss = 0.45833160\n",
      "Iteration 80, loss = 0.45673064\n",
      "Iteration 81, loss = 0.45643719\n",
      "Iteration 82, loss = 0.45589729\n",
      "Iteration 83, loss = 0.45497622\n",
      "Iteration 84, loss = 0.45507657\n",
      "Iteration 85, loss = 0.45400356\n",
      "Iteration 86, loss = 0.45339288\n",
      "Iteration 87, loss = 0.45203971\n",
      "Iteration 88, loss = 0.45162649\n",
      "Iteration 89, loss = 0.45095862\n",
      "Iteration 90, loss = 0.45085199\n",
      "Iteration 91, loss = 0.45137972\n",
      "Iteration 92, loss = 0.44981982\n",
      "Iteration 93, loss = 0.44900313\n",
      "Iteration 94, loss = 0.44787559\n",
      "Iteration 95, loss = 0.44722527\n",
      "Iteration 96, loss = 0.44704590\n",
      "Iteration 97, loss = 0.44669458\n",
      "Iteration 98, loss = 0.44560679\n",
      "Iteration 99, loss = 0.44606611\n",
      "Iteration 100, loss = 0.44476003\n",
      "Iteration 101, loss = 0.44491535\n",
      "Iteration 102, loss = 0.44337468\n",
      "Iteration 103, loss = 0.44403367\n",
      "Iteration 104, loss = 0.44335664\n",
      "Iteration 105, loss = 0.44228774\n",
      "Iteration 106, loss = 0.44165937\n",
      "Iteration 107, loss = 0.44171247\n",
      "Iteration 108, loss = 0.44085041\n",
      "Iteration 109, loss = 0.44160576\n",
      "Iteration 110, loss = 0.43952883\n",
      "Iteration 111, loss = 0.44050462\n",
      "Iteration 112, loss = 0.43909972\n",
      "Iteration 113, loss = 0.43966621\n",
      "Iteration 114, loss = 0.43827853\n",
      "Iteration 115, loss = 0.43831839\n",
      "Iteration 116, loss = 0.43765615\n",
      "Iteration 117, loss = 0.43822577\n",
      "Iteration 118, loss = 0.43723668\n",
      "Iteration 119, loss = 0.43706877\n",
      "Iteration 120, loss = 0.43638154\n",
      "Iteration 121, loss = 0.43601589\n",
      "Iteration 122, loss = 0.43552676\n",
      "Iteration 123, loss = 0.43578402\n",
      "Iteration 124, loss = 0.43548834\n",
      "Iteration 125, loss = 0.43520768\n",
      "Iteration 126, loss = 0.43422144\n",
      "Iteration 127, loss = 0.43431074\n",
      "Iteration 128, loss = 0.43558001\n",
      "Iteration 129, loss = 0.43333936\n",
      "Iteration 130, loss = 0.43382487\n",
      "Iteration 131, loss = 0.43353531\n",
      "Iteration 132, loss = 0.43297493\n",
      "Iteration 133, loss = 0.43246689\n",
      "Iteration 134, loss = 0.43187858\n",
      "Iteration 135, loss = 0.43391505\n",
      "Iteration 136, loss = 0.43157439\n",
      "Iteration 137, loss = 0.43161135\n",
      "Iteration 138, loss = 0.43127433\n",
      "Iteration 139, loss = 0.43234142\n",
      "Iteration 140, loss = 0.43089933\n",
      "Iteration 141, loss = 0.43028184\n",
      "Iteration 142, loss = 0.43137114\n",
      "Iteration 143, loss = 0.43062682\n",
      "Iteration 144, loss = 0.43108714\n",
      "Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.\n",
      "RF K-fold: 2 accuracy score is 0.780899 \n",
      "Iteration 1, loss = 0.93008152\n",
      "Iteration 2, loss = 0.80371565\n",
      "Iteration 3, loss = 0.76953899\n",
      "Iteration 4, loss = 0.75040334\n",
      "Iteration 5, loss = 0.72955950\n",
      "Iteration 6, loss = 0.71026217\n",
      "Iteration 7, loss = 0.69280704\n",
      "Iteration 8, loss = 0.67585460\n",
      "Iteration 9, loss = 0.66205208\n",
      "Iteration 10, loss = 0.64820799\n",
      "Iteration 11, loss = 0.63628863\n",
      "Iteration 12, loss = 0.62669976\n",
      "Iteration 13, loss = 0.61762786\n",
      "Iteration 14, loss = 0.60981834\n",
      "Iteration 15, loss = 0.60234279\n",
      "Iteration 16, loss = 0.59524083\n",
      "Iteration 17, loss = 0.58896943\n",
      "Iteration 18, loss = 0.58310865\n",
      "Iteration 19, loss = 0.57737308\n",
      "Iteration 20, loss = 0.57212613\n",
      "Iteration 21, loss = 0.56640615\n",
      "Iteration 22, loss = 0.56154174\n",
      "Iteration 23, loss = 0.55732142\n",
      "Iteration 24, loss = 0.55510046\n",
      "Iteration 25, loss = 0.54848957\n",
      "Iteration 26, loss = 0.54591574\n",
      "Iteration 27, loss = 0.54249309\n",
      "Iteration 28, loss = 0.53941493\n",
      "Iteration 29, loss = 0.53426447\n",
      "Iteration 30, loss = 0.53036971\n",
      "Iteration 31, loss = 0.52733488\n",
      "Iteration 32, loss = 0.52435833\n",
      "Iteration 33, loss = 0.52116605\n",
      "Iteration 34, loss = 0.51826425\n",
      "Iteration 35, loss = 0.51671375\n",
      "Iteration 36, loss = 0.51342076\n",
      "Iteration 37, loss = 0.51175563\n",
      "Iteration 38, loss = 0.50856888\n",
      "Iteration 39, loss = 0.50645094\n",
      "Iteration 40, loss = 0.50459756\n",
      "Iteration 41, loss = 0.50221851\n",
      "Iteration 42, loss = 0.50033292\n",
      "Iteration 43, loss = 0.49797005\n",
      "Iteration 44, loss = 0.49745234\n",
      "Iteration 45, loss = 0.49510947\n",
      "Iteration 46, loss = 0.49350126\n",
      "Iteration 47, loss = 0.49086406\n",
      "Iteration 48, loss = 0.48928779\n",
      "Iteration 49, loss = 0.48782742\n",
      "Iteration 50, loss = 0.48644448\n",
      "Iteration 51, loss = 0.48566941\n",
      "Iteration 52, loss = 0.48388540\n",
      "Iteration 53, loss = 0.48258432\n",
      "Iteration 54, loss = 0.48157919\n",
      "Iteration 55, loss = 0.47954402\n",
      "Iteration 56, loss = 0.47830771\n",
      "Iteration 57, loss = 0.47684302\n",
      "Iteration 58, loss = 0.47621719\n",
      "Iteration 59, loss = 0.47455781\n",
      "Iteration 60, loss = 0.47342473\n",
      "Iteration 61, loss = 0.47236768\n",
      "Iteration 62, loss = 0.47118983\n",
      "Iteration 63, loss = 0.47037915\n",
      "Iteration 64, loss = 0.46955327\n",
      "Iteration 65, loss = 0.46876730\n",
      "Iteration 66, loss = 0.46799017\n",
      "Iteration 67, loss = 0.46635727\n",
      "Iteration 68, loss = 0.46579211\n",
      "Iteration 69, loss = 0.46446875\n",
      "Iteration 70, loss = 0.46580857\n",
      "Iteration 71, loss = 0.46316948\n",
      "Iteration 72, loss = 0.46212425\n",
      "Iteration 73, loss = 0.46104990\n",
      "Iteration 74, loss = 0.46047650\n",
      "Iteration 75, loss = 0.45956136\n",
      "Iteration 76, loss = 0.45943422\n",
      "Iteration 77, loss = 0.45842594\n",
      "Iteration 78, loss = 0.45769430\n",
      "Iteration 79, loss = 0.45690823\n",
      "Iteration 80, loss = 0.45652201\n",
      "Iteration 81, loss = 0.45548438\n",
      "Iteration 82, loss = 0.45596358\n",
      "Iteration 83, loss = 0.45500590\n",
      "Iteration 84, loss = 0.45338304\n",
      "Iteration 85, loss = 0.45365765\n",
      "Iteration 86, loss = 0.45276196\n",
      "Iteration 87, loss = 0.45198751\n",
      "Iteration 88, loss = 0.45139031\n",
      "Iteration 89, loss = 0.45091978\n",
      "Iteration 90, loss = 0.45060993\n",
      "Iteration 91, loss = 0.44955688\n",
      "Iteration 92, loss = 0.44895508\n",
      "Iteration 93, loss = 0.44858366\n",
      "Iteration 94, loss = 0.44871482\n",
      "Iteration 95, loss = 0.44861123\n",
      "Iteration 96, loss = 0.44697657\n",
      "Iteration 97, loss = 0.44680195\n",
      "Iteration 98, loss = 0.44718115\n",
      "Iteration 99, loss = 0.44588459\n",
      "Iteration 100, loss = 0.44604141\n",
      "Iteration 101, loss = 0.44616406\n",
      "Iteration 102, loss = 0.44481093\n",
      "Iteration 103, loss = 0.44441079\n",
      "Iteration 104, loss = 0.44519828\n",
      "Iteration 105, loss = 0.44324109\n",
      "Iteration 106, loss = 0.44490024\n",
      "Iteration 107, loss = 0.44249643\n",
      "Iteration 108, loss = 0.44262379\n",
      "Iteration 109, loss = 0.44251668\n",
      "Iteration 110, loss = 0.44194046\n",
      "Iteration 111, loss = 0.44102552\n",
      "Iteration 112, loss = 0.44059039\n",
      "Iteration 113, loss = 0.44167546\n",
      "Iteration 114, loss = 0.43995577\n",
      "Iteration 115, loss = 0.43983287\n",
      "Iteration 116, loss = 0.43946049\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iteration 117, loss = 0.44024085\n",
      "Iteration 118, loss = 0.43886826\n",
      "Iteration 119, loss = 0.44168026\n",
      "Iteration 120, loss = 0.44060943\n",
      "Iteration 121, loss = 0.43779033\n",
      "Iteration 122, loss = 0.43799605\n",
      "Iteration 123, loss = 0.43732623\n",
      "Iteration 124, loss = 0.43746177\n",
      "Iteration 125, loss = 0.43670172\n",
      "Iteration 126, loss = 0.43644253\n",
      "Iteration 127, loss = 0.43650152\n",
      "Iteration 128, loss = 0.43648184\n",
      "Iteration 129, loss = 0.43634905\n",
      "Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.\n",
      "RF K-fold: 3 accuracy score is 0.820225 \n",
      "Iteration 1, loss = 0.92051348\n",
      "Iteration 2, loss = 0.79753057\n",
      "Iteration 3, loss = 0.76746464\n",
      "Iteration 4, loss = 0.74950421\n",
      "Iteration 5, loss = 0.72947996\n",
      "Iteration 6, loss = 0.70943067\n",
      "Iteration 7, loss = 0.69049381\n",
      "Iteration 8, loss = 0.67386197\n",
      "Iteration 9, loss = 0.65745330\n",
      "Iteration 10, loss = 0.64396128\n",
      "Iteration 11, loss = 0.63197588\n",
      "Iteration 12, loss = 0.62114819\n",
      "Iteration 13, loss = 0.61304679\n",
      "Iteration 14, loss = 0.60448256\n",
      "Iteration 15, loss = 0.59757136\n",
      "Iteration 16, loss = 0.59072511\n",
      "Iteration 17, loss = 0.58384819\n",
      "Iteration 18, loss = 0.57909786\n",
      "Iteration 19, loss = 0.57308343\n",
      "Iteration 20, loss = 0.56778729\n",
      "Iteration 21, loss = 0.56284962\n",
      "Iteration 22, loss = 0.55825713\n",
      "Iteration 23, loss = 0.55294349\n",
      "Iteration 24, loss = 0.54922632\n",
      "Iteration 25, loss = 0.54514066\n",
      "Iteration 26, loss = 0.54136718\n",
      "Iteration 27, loss = 0.53827498\n",
      "Iteration 28, loss = 0.53426777\n",
      "Iteration 29, loss = 0.53130097\n",
      "Iteration 30, loss = 0.52817207\n",
      "Iteration 31, loss = 0.52494872\n",
      "Iteration 32, loss = 0.52196430\n",
      "Iteration 33, loss = 0.51957006\n",
      "Iteration 34, loss = 0.51672756\n",
      "Iteration 35, loss = 0.51442993\n",
      "Iteration 36, loss = 0.51194959\n",
      "Iteration 37, loss = 0.51044354\n",
      "Iteration 38, loss = 0.50885648\n",
      "Iteration 39, loss = 0.50608544\n",
      "Iteration 40, loss = 0.50481570\n",
      "Iteration 41, loss = 0.50207147\n",
      "Iteration 42, loss = 0.49983951\n",
      "Iteration 43, loss = 0.49833461\n",
      "Iteration 44, loss = 0.49673579\n",
      "Iteration 45, loss = 0.49535527\n",
      "Iteration 46, loss = 0.49418335\n",
      "Iteration 47, loss = 0.49260148\n",
      "Iteration 48, loss = 0.49047249\n",
      "Iteration 49, loss = 0.48898949\n",
      "Iteration 50, loss = 0.48757242\n",
      "Iteration 51, loss = 0.48646800\n",
      "Iteration 52, loss = 0.48469044\n",
      "Iteration 53, loss = 0.48452000\n",
      "Iteration 54, loss = 0.48522260\n",
      "Iteration 55, loss = 0.48273680\n",
      "Iteration 56, loss = 0.48042093\n",
      "Iteration 57, loss = 0.47905286\n",
      "Iteration 58, loss = 0.47853104\n",
      "Iteration 59, loss = 0.47680884\n",
      "Iteration 60, loss = 0.47599954\n",
      "Iteration 61, loss = 0.47520179\n",
      "Iteration 62, loss = 0.47399810\n",
      "Iteration 63, loss = 0.47322422\n",
      "Iteration 64, loss = 0.47221926\n",
      "Iteration 65, loss = 0.47235435\n",
      "Iteration 66, loss = 0.47180397\n",
      "Iteration 67, loss = 0.46947967\n",
      "Iteration 68, loss = 0.46932729\n",
      "Iteration 69, loss = 0.46841593\n",
      "Iteration 70, loss = 0.46809210\n",
      "Iteration 71, loss = 0.46711991\n",
      "Iteration 72, loss = 0.46646914\n",
      "Iteration 73, loss = 0.46558016\n",
      "Iteration 74, loss = 0.46427450\n",
      "Iteration 75, loss = 0.46488738\n",
      "Iteration 76, loss = 0.46340618\n",
      "Iteration 77, loss = 0.46236088\n",
      "Iteration 78, loss = 0.46256938\n",
      "Iteration 79, loss = 0.46116601\n",
      "Iteration 80, loss = 0.46172538\n",
      "Iteration 81, loss = 0.46062484\n",
      "Iteration 82, loss = 0.45944479\n",
      "Iteration 83, loss = 0.45962275\n",
      "Iteration 84, loss = 0.45819061\n",
      "Iteration 85, loss = 0.45861037\n",
      "Iteration 86, loss = 0.45722436\n",
      "Iteration 87, loss = 0.45679582\n",
      "Iteration 88, loss = 0.45667347\n",
      "Iteration 89, loss = 0.45649553\n",
      "Iteration 90, loss = 0.45520550\n",
      "Iteration 91, loss = 0.45527114\n",
      "Iteration 92, loss = 0.45420625\n",
      "Iteration 93, loss = 0.45394977\n",
      "Iteration 94, loss = 0.45370338\n",
      "Iteration 95, loss = 0.45327335\n",
      "Iteration 96, loss = 0.45312960\n",
      "Iteration 97, loss = 0.45284545\n",
      "Iteration 98, loss = 0.45207631\n",
      "Iteration 99, loss = 0.45161524\n",
      "Iteration 100, loss = 0.45124391\n",
      "Iteration 101, loss = 0.45105049\n",
      "Iteration 102, loss = 0.45054790\n",
      "Iteration 103, loss = 0.45090850\n",
      "Iteration 104, loss = 0.45151647\n",
      "Iteration 105, loss = 0.45005931\n",
      "Iteration 106, loss = 0.44895859\n",
      "Iteration 107, loss = 0.44901630\n",
      "Iteration 108, loss = 0.44967120\n",
      "Iteration 109, loss = 0.44791424\n",
      "Iteration 110, loss = 0.44819231\n",
      "Iteration 111, loss = 0.44782501\n",
      "Iteration 112, loss = 0.44842040\n",
      "Training loss did not improve more than tol=0.000100 for two consecutive epochs. Stopping.\n",
      "RF K-fold: 4 accuracy score is 0.820225 \n",
      "Training is complete\n"
     ]
    }
   ],
   "source": [
    "# Create our OOF train and test predictions. These base results will be used as new features\n",
    "nn_oof_train0, nn_oof_test0 = get_oof(NN,x_train, y_train, x_test)\n",
    "print(\"Training is complete\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.800224466891\n"
     ]
    }
   ],
   "source": [
    "#print(rf_oof_train.reshape(-1))\n",
    "print(accuracy_score(y_train,NN.predict(x_train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#直接某个基学习器\n",
    "#print(rf_oof_test.round())\n",
    "#StackingSubmission = pd.DataFrame({ 'PassengerId': PassengerId,\n",
    "#                           'Survived': rf_oof_test.round().reshape(-1).astype(np.int8)})\n",
    "#StackingSubmission.to_csv(\"RFSubmission.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
